{
 "cells": [
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading worldtree...\n",
      "Downloading and preparing dataset worldtree_dataset/thoughtsource to /Users/robertpraas/.cache/huggingface/datasets/worldtree_dataset/thoughtsource/1.0.0/4ec0cd827b41f05891af9a27bf461fecd407e2fe7c1beebfed1eb00193c2cd52...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7a07bcfd9c9142eb85e693d17d0f168a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Generating train split: 0 examples [00:00, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7518e59bd878416285067089281aa4a1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Generating test split: 0 examples [00:00, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "441adf5dbe24426e88dc042823fb74a0",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Generating validation split: 0 examples [00:00, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Dataset worldtree_dataset downloaded and prepared to /Users/robertpraas/.cache/huggingface/datasets/worldtree_dataset/thoughtsource/1.0.0/4ec0cd827b41f05891af9a27bf461fecd407e2fe7c1beebfed1eb00193c2cd52. Subsequent calls will reuse this data.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2b18008614fb43f3980f98b9f8064692",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/3 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from cot import Collection\n",
    "coll = Collection(\"worldtree\", load_pregenerated_cots=True,generate_mode=\"recache\") #does not load pre-generated cots"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [],
   "source": [
    "coll = coll.select(split=\"test\", number_samples=2)\n",
    "#coll.select_generated_cots(cot_trigger = \"kojima-01\")\n",
    "#coll.select_generated_cots(api_service='cohere')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "| Name      | Train   | Valid   |   Test |\n",
       "|-----------|---------|---------|--------|\n",
       "| worldtree | -       | -       |      2 |\n",
       "\n",
       "Not loaded: ['aqua', 'asdiv', 'commonsense_qa', 'entailment_bank', 'gsm8k', 'mawps', 'med_qa', 'med_qa_open', 'medmc_qa', '_init_', 'mmlu_clinical_knowledge', 'mmlu_college_biology', 'mmlu_college_medicine', 'mmlu_medical_genetics', 'mmlu_professional_medicine', '_init_', 'mmlu_anatomy', 'open_book_qa', 'pubmed_qa', 'qed', 'strategy_qa', 'svamp']"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "coll"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'id': 'worldtree_test_0',\n",
       " 'ref_id': '',\n",
       " 'question': 'What remains in the same location in the sky of the Northern Hemisphere each night?',\n",
       " 'type': 'multiplechoice',\n",
       " 'choices': ['the Sun', 'the Little Dipper', 'the North Star', 'the Moon'],\n",
       " 'context': '',\n",
       " 'cot': ['The north star does not move every night in the sky in the northern hemisphere.',\n",
       "  'Moving changes position.',\n",
       "  'To remain means to not change.',\n",
       "  'If something does not move then that something can be found in the same location.',\n",
       "  'Place is synonymous with position.'],\n",
       " 'answer': ['the North Star'],\n",
       " 'generated_cot': [],\n",
       " 'feedback': []}"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "coll['worldtree']['test'][0]"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Generate and extract"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "from langchain.llms import OpenAI \n",
    "from langchain.prompts import PromptTemplate \n",
    "from langchain.chains.llm import LLMChain\n",
    "from langchain.chat_models import ChatOpenAI\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"CoT Chain\"\"\"\n",
    "\n",
    "llm = ChatOpenAI(temperature=.0,model_name=\"gpt-3.5-turbo\")\n",
    "\n",
    "template = \"\"\"{instruction}\n",
    "\n",
    "Question: {question}\n",
    "Answer_choices: {answer_choices}\n",
    "\n",
    "{cot_trigger}\n",
    "\"\"\"\n",
    "\n",
    "prompt_template = PromptTemplate(input_variables=[\"instruction\",\"question\",\"answer_choices\",\"cot_trigger\"], template=template)\n",
    "cot_chain = LLMChain(llm=llm, prompt=prompt_template,output_key=\"cot\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"answer extraction chain\"\"\"\n",
    "\n",
    "extraction_template = \"\"\"{instruction}\n",
    "\n",
    "Question: {question}\n",
    "Answer_choices: {answer_choices}\n",
    "\n",
    "Cot: {cot_trigger}{cot}\n",
    "{answer_extraction}\n",
    "\"\"\"\n",
    "\n",
    "prompt_template = PromptTemplate(input_variables=[\"instruction\",\"question\",\"answer_choices\",\"cot_trigger\",\"cot\",\"answer_extraction\"], template=extraction_template)\n",
    "answer_chain = LLMChain(llm=llm, prompt=prompt_template,output_key=\"predicted_answer\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"CoT-Ans_extraction chain\"\"\"\n",
    "\n",
    "from langchain.chains import SequentialChain\n",
    "overall_chain = SequentialChain(chains=[cot_chain, answer_chain],\n",
    "                                input_variables=[\"instruction\",\"question\",\"answer_choices\",\"cot_trigger\",\"answer_extraction\"],\n",
    "                                output_variables=[\"cot\", \"predicted_answer\"],\n",
    "                                verbose=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Generating worldtree...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ae038ed3508847fe86efcb2bbf4acf06",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/2 [00:00<?, ?ex/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "\u001b[1m> Entering new SequentialChain chain...\u001b[0m\n",
      "\n",
      "\u001b[1m> Finished chain.\u001b[0m\n",
      "{'id': 'worldtree_test_788', 'ref_id': '', 'question': 'When light hits a mirror, most of the light is', 'type': 'multiplechoice', 'choices': ['refracted.', 'reflected.', 'absorbed.', 'transmitted.'], 'context': '', 'cot': ['A mirror reflects light.', 'Reflection is when a wave bounces off a surface and travels in the opposite direction relative to the angle of incidence.', 'Light is a kind of wave.', 'When light hits a reflective object , that light bounces off that object.', 'A mirror is a kind of reflective object.'], 'answer': ['reflected.'], 'generated_cot': [{'id': '17eb19ff-79ba-4b6a-abe8-31b6b854be75', 'fragments_version': None, 'instruction': 'Be faithful and a little hopeful', 'cot_trigger': \"Answer: Let's think step by step.\", 'cot_trigger_template': '', 'prompt_text': '', 'cot': 'When light hits a mirror, it bounces back off the surface of the mirror. This process is called reflection. Therefore, the correct answer is B) reflected.', 'answers': [{'id': 'ab70f901-9d0a-494b-999f-11ad0939fc68', 'answer_extraction': 'Therefore, among A through D, the answer is', 'answer_extraction_template': '', 'answer_extraction_text': '', 'answer': 'B) reflected.', 'answer_from_choices': '', 'correct_answer': None}], 'author': '', 'date': '2023/06/12 12:06:28', 'api_service': '', 'model': \"{'name': 'gpt-3.5-turbo', 'temperature': 0, 'max_tokens': 800}\", 'comment': 'generated and extracted', 'annotations': []}], 'feedback': []}\n",
      "\n",
      "\n",
      "\u001b[1m> Entering new SequentialChain chain...\u001b[0m\n",
      "\n",
      "\u001b[1m> Finished chain.\u001b[0m\n",
      "{'id': 'worldtree_test_1552', 'ref_id': '', 'question': 'Which of these is a chemical property of a material?', 'type': 'multiplechoice', 'choices': ['density', 'flammability', 'color', 'brittleness'], 'context': '', 'cot': ['Flammability is a kind of chemical property.', 'Different materials usually have different properties.'], 'answer': ['flammability'], 'generated_cot': [{'id': '6f5776f7-e59e-4777-9b89-cb411123c301', 'fragments_version': None, 'instruction': 'Be faithful and a little hopeful', 'cot_trigger': \"Answer: Let's think step by step.\", 'cot_trigger_template': '', 'prompt_text': '', 'cot': 'A) Density is a physical property, as it describes the amount of mass in a given volume.\\nB) Flammability is a chemical property, as it describes how easily a material will ignite and burn.\\nC) Color is a physical property, as it describes the way a material reflects or absorbs light.\\nD) Brittleness is a physical property, as it describes how easily a material will break or shatter.\\n\\nTherefore, the correct answer is B) flammability.', 'answers': [{'id': 'f4934b44-c1a0-4cdf-a9ab-81ee3f0ff3cf', 'answer_extraction': 'Therefore, among A through D, the answer is', 'answer_extraction_template': '', 'answer_extraction_text': '', 'answer': 'B) flammability.', 'answer_from_choices': '', 'correct_answer': None}], 'author': '', 'date': '2023/06/12 12:06:34', 'api_service': '', 'model': \"{'name': 'gpt-3.5-turbo', 'temperature': 0, 'max_tokens': 800}\", 'comment': 'generated and extracted', 'annotations': []}], 'feedback': []}\n"
     ]
    }
   ],
   "source": [
    "\"\"\"Generate CoT with use of TS-schema\"\"\"\n",
    "#compare with config used before; what about max tokens?\n",
    "#config contains what the chain needs\n",
    "input_dict = {'input_dict':\n",
    "              {\n",
    "                  'chain': overall_chain,\n",
    "                  \"instruction\": \"Be faithful and a little hopeful\",\n",
    "                  \"cot_trigger\": \"Answer: Let's think step by step.\",\n",
    "                  \"answer_extraction\": \"Therefore, among A through D, the answer is\",\n",
    "                  'model': \"gpt-3.5-turbo\",\n",
    "                  'temperature': 0,\n",
    "                  'max_tokens': 800\n",
    "              }\n",
    "              }\n",
    "\n",
    "coll.generate_extract_flexible(input_dict=input_dict)\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'id': '7a42d609-d5c7-4272-865a-6822b9ed267f',\n",
       " 'fragments_version': None,\n",
       " 'instruction': 'Be faithful and a little hopeful',\n",
       " 'cot_trigger': \"Answer: Let's think step by step.\",\n",
       " 'cot_trigger_template': '',\n",
       " 'prompt_text': '',\n",
       " 'cot': 'When light hits a mirror, it bounces back off the surface of the mirror. This process is called reflection. Therefore, the correct answer is B) reflected.',\n",
       " 'answers': [{'id': '14b14f65-4326-47d3-a91c-fbb3b84a9ee2',\n",
       "   'answer_extraction': 'Therefore, among A through D, the answer is',\n",
       "   'answer_extraction_template': '',\n",
       "   'answer_extraction_text': '',\n",
       "   'answer': 'B) reflected.',\n",
       "   'answer_from_choices': '',\n",
       "   'correct_answer': None}],\n",
       " 'author': '',\n",
       " 'date': '2023/06/12 11:47:59',\n",
       " 'api_service': '',\n",
       " 'model': \"{'name': 'gpt-3.5-turbo', 'temperature': 0, 'max_tokens': 800}\",\n",
       " 'comment': 'generated and extracted',\n",
       " 'annotations': []}"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "coll['worldtree']['test'][0]['generated_cot'][0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5dd10baccced4dd48168c8bec3746431",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/2 [00:00<?, ?ex/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "{'worldtree': {'test': {'accuracy': {'gpt-3.5-turbo': {'hopeful_Kojima': 1.0}}}}}"
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Title for CoT trigger and extraction used\n",
    "coll.evaluate(title='hopeful_Kojima')"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Generate or extract"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [],
   "source": [
    "#from langchain.llms.openai import OpenAIChat\n",
    "\n",
    "from langchain.llms import OpenAI \n",
    "from langchain.prompts import PromptTemplate \n",
    "from langchain.chains.llm import LLMChain\n",
    "from cot import Collection\n",
    "import json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Generating worldtree...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ae65c6ef3653497da8e2c5d9a82498f9",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/2 [00:00<?, ?ex/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[{'id': '14b14f65-4326-47d3-a91c-fbb3b84a9ee2', 'answer_extraction': 'Therefore, among A through D, the answer is', 'answer_extraction_template': '', 'answer_extraction_text': '', 'answer': 'B) reflected.', 'answer_from_choices': 'B', 'correct_answer': True}]\n",
      "################\n",
      "[{'id': '14b14f65-4326-47d3-a91c-fbb3b84a9ee2', 'answer_extraction': 'Therefore, among A through D, the answer is', 'answer_extraction_template': '', 'answer_extraction_text': '', 'answer': 'B) reflected.', 'answer_from_choices': 'B', 'correct_answer': True}, {'id': '900c7cda-1d07-4092-9d6c-48226fce1cab', 'answer_extraction': 'Therefore, among A through D, the answer is', 'answer_extraction_template': '', 'answer_extraction_text': '', 'answer': 'B) reflected.', 'answer_from_choices': '', 'correct_answer': None}]\n",
      "[{'id': '8ac394a9-59d6-4433-935a-4d6b8d84671a', 'answer_extraction': 'Therefore, among A through D, the answer is', 'answer_extraction_template': '', 'answer_extraction_text': '', 'answer': 'B) flammability.', 'answer_from_choices': 'B', 'correct_answer': True}]\n",
      "################\n",
      "[{'id': '8ac394a9-59d6-4433-935a-4d6b8d84671a', 'answer_extraction': 'Therefore, among A through D, the answer is', 'answer_extraction_template': '', 'answer_extraction_text': '', 'answer': 'B) flammability.', 'answer_from_choices': 'B', 'correct_answer': True}, {'id': '7c055504-6faa-4288-bb45-dfb67f28ceb7', 'answer_extraction': 'Therefore, among A through D, the answer is', 'answer_extraction_template': '', 'answer_extraction_text': '', 'answer': 'B) flammability.', 'answer_from_choices': '', 'correct_answer': None}]\n"
     ]
    }
   ],
   "source": [
    "\"\"\"Extract script: Assumes there are CoTs in the dataset already\"\"\"\n",
    "\n",
    "input_dict = {\n",
    "    'input_dict': {\n",
    "        'chain': answer_chain,\n",
    "        \"instruction\": \"Be faithful and a little hopeful\",\n",
    "        \"cot_trigger\": \"Answer: Let's think step by step.\",\n",
    "        \"answer_extraction\": \"Therefore, among A through D, the answer is\",\n",
    "        'model': \"gpt-3.5-turbo\",\n",
    "        'api_service': 'OpenAI',\n",
    "        'temperature': 0,\n",
    "        'max_tokens': 800\n",
    "    }\n",
    "} \n",
    "coll.extract_flexible(input_dict=input_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Generating worldtree...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "836b6d1152484b4591ac676cd7428436",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/2 [00:00<?, ?ex/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "\"\"\"Generate only\"\"\"\n",
    "input_dict = {\n",
    "    'input_dict': {\n",
    "        'chain': cot_chain,\n",
    "        \"instruction\": \"Be faithful and a little hopeful\",\n",
    "        \"cot_trigger\": \"Answer: Let's think step by step.\",\n",
    "        \"answer_extraction\": \"Therefore, among A through D, the answer is\",\n",
    "        'model': \"gpt-3.5-turbo\",\n",
    "        'api_service': 'OpenAI',\n",
    "        'temperature': 0,\n",
    "        'max_tokens': 800\n",
    "    }\n",
    "} \n",
    "coll.generate_flexible(input_dict=input_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "coll['worldtree']['test'][0]['generated_cot'][2]"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Reflection"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [],
   "source": [
    "llm = ChatOpenAI(temperature=.0,model_name=\"gpt-3.5-turbo\")\n",
    "\n",
    "reflect_template = \"\"\"\n",
    "    Question: {question}\n",
    "    Answer_choices: {answer_choices}\n",
    "\n",
    "    Cot: {cot_trigger}{cot}\n",
    "    {answer_extraction}\n",
    "    Answer: {answer}\n",
    "    \n",
    "    {reflection_prompt}\n",
    "    \"\"\"\n",
    "prompt_template = PromptTemplate(input_variables=[\"question\",\"answer_choices\",\"cot_trigger\",\"cot\",\"answer_extraction\",'answer','reflection_prompt'], template=reflect_template)\n",
    "reflect_chain = LLMChain(llm=llm, prompt=prompt_template,output_key=\"reflection\")\n",
    "\n",
    "extraction_template = \"\"\"{instruction}\n",
    "\n",
    "    Question: {question}\n",
    "    Answer_choices: {answer_choices}\n",
    "\n",
    "    Cot: {cot_trigger}{cot}\n",
    "    {answer_extraction}{answer}\n",
    "    {reflection_prompt}{reflection}\n",
    "\n",
    "    {reflect_answer_extraction}\n",
    "    \"\"\"\n",
    "    #Get reflection\n",
    "prompt_template = PromptTemplate(input_variables=[\"instruction\",\"question\",\"answer_choices\",\"cot_trigger\",\"cot\",\"answer_extraction\",'answer','reflection_prompt','reflection','reflect_answer_extraction'], template=extraction_template)\n",
    "reflect_answer_chain = LLMChain(llm=llm, prompt=prompt_template,output_key=\"reflection_answer\")\n",
    "\n",
    "    # This is the overall chain where we run these two chains in sequence.\n",
    "from langchain.chains import SequentialChain\n",
    "reflect_overall_chain = SequentialChain(chains=[reflect_chain, reflect_answer_chain],input_variables=[\"instruction\",\"question\",\"answer_choices\",\"cot_trigger\",\"answer_extraction\",'cot','answer','reflection_prompt','reflect_answer_extraction'],\n",
    "        output_variables=[\"reflection\", \"reflection_answer\"],\n",
    "        verbose=True)\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Generating worldtree...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "dfa5a93989864dae87b622137d07ebe1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/2 [00:00<?, ?ex/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "\u001b[1m> Entering new SequentialChain chain...\u001b[0m\n",
      "\n",
      "\u001b[1m> Finished chain.\u001b[0m\n",
      "\n",
      "\n",
      "\u001b[1m> Entering new SequentialChain chain...\u001b[0m\n",
      "\n",
      "\u001b[1m> Finished chain.\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "#check for what is already in item\n",
    "input_dict = {'input_dict':{\n",
    "    'chain':reflect_overall_chain,\n",
    "    'cot_trigger':\"\", \n",
    "    'answer':\"\", \n",
    "    'answer_extraction': \"\", \n",
    "    'cot': \"\", \n",
    "    'instruction': \"\",\n",
    "    'api_service': \"chat_openai\", \n",
    "    'model': \"gpt-3.5-turbo\",\n",
    "    'reflection_prompt':\"Double check this\",\n",
    "    'reflect_answer_extraction':'Based on the reflection, what is the definite answer?',\n",
    "    'temperature': 0,\n",
    "    'max_tokens': 800 \n",
    "}\n",
    "}\n",
    "coll.metareason_flexible(input_dict=input_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'id': '530e51f1-39d2-4fa5-a586-efba3dfa753e',\n",
       " 'fragments_version': '',\n",
       " 'instruction': '',\n",
       " 'cot_trigger': 'Double check this',\n",
       " 'cot_trigger_template': '',\n",
       " 'prompt_text': '',\n",
       " 'cot': 'As an AI language model, I can confirm that the answer is correct. When light hits a mirror, it is reflected back off the surface of the mirror. This is due to the smooth surface of the mirror, which allows the light to bounce back in a predictable way. Therefore, the correct answer is B) reflected.',\n",
       " 'answers': [{'id': 'acae3361-0db6-44c9-a51d-cabe875c9a33',\n",
       "   'answer_extraction': 'Based on the reflection, what is the definite answer?',\n",
       "   'answer_extraction_template': '',\n",
       "   'answer_extraction_text': 'self_reflection',\n",
       "   'answer': 'The definite answer is B) reflected.',\n",
       "   'answer_from_choices': '',\n",
       "   'correct_answer': None}],\n",
       " 'author': '',\n",
       " 'date': '2023/06/12 12:06:42',\n",
       " 'api_service': 'chat_openai',\n",
       " 'model': \"{'name': 'gpt-3.5-turbo', 'temperature': 0, 'max_tokens': 800}\",\n",
       " 'comment': 'self_reflection cot',\n",
       " 'annotations': []}"
      ]
     },
     "execution_count": 74,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "coll['worldtree']['test'][0]['generated_cot'][1]"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "base",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.16"
  },
  "vscode": {
   "interpreter": {
    "hash": "40d3a090f54c6569ab1632332b64b2c03c39dcf918b08424e98f38b5ae0af88f"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}
